import sys
from collections import defaultdict
from math import sqrt

user_key = defaultdict(list)
key_count = defaultdict(int)
common_keys = []

num_items = 6095
max_num_keys = 1 << 30
only_ikey = False

user_profile = False
item_ridx = False
user_key_word = False

def proc_unweighted_keyw(keyw):
    if not keyw:
        return list()

    d = defaultdict(float)
    for k in keyw:
        d[k] += 1
    r = 1.0 / sqrt(len(d))
    return map(lambda (k, c): (k, c*r), d.iteritems())
    
def load_keyw(fname):
    for line in open(fname):
        if user_key_word:
            uid, keyw = line.strip().split()
            uid = int(uid)
            if uid >= num_items and only_ikey:
                continue

            keyw = map(lambda s: s.split(':'), keyw.split(';'))
            keyw = map(lambda (fid, v): (int(fid), float(v)), keyw)
        elif user_profile:
            uid, birth, gender, retweet, keyw = line.strip().split()
            uid = int(uid)
            if uid >= num_items and only_ikey:
                continue

            keyw = map(int, keyw.split(';'))
            keyw = proc_unweighted_keyw(keyw)
        elif item_ridx:
            uid, cat, keyw = line.strip().split()
            uid = int(uid)
            if uid >= num_items and only_ikey:
                continue

            keyw = map(int, keyw.split(';'))
            keyw = proc_unweighted_keyw(keyw)

        user_key[uid] = keyw
        for k, w in keyw:
            key_count[k] += 1
        
    key_list = key_count.keys()
    key_list.sort(key=lambda k: -key_count[k])
    for k in key_list[0:max_num_keys]:
        common_keys.append(k)

def print_keyw(ofname):
    ofile = open(ofname, "w")
    for n, k in enumerate(common_keys):
        ofile.write('%d %d %d\n' % (n, k, key_count[k]))
    ofile.close()


if __name__ == '__main__':
    if len(sys.argv) < 4:
        print 'usage: <ukeyw> <common_keys.out> <max_num_keys> [--only_ikey]'
        print 'learn user_key_word/item/user_profile from file names'
        sys.exit(1)
    max_num_keys = int(sys.argv[3])

    for arg in sys.argv[4:]:
        assert arg in ["--only_ikey"]
        globals()[arg[2:]] = True
    
    keywf = sys.argv[1]
    user_profile = keywf.find('user_profile') >= 0
    item_ridx = keywf.find('item') >= 0
    user_key_word = keywf.find('user_key_word') >= 0

    assert user_profile + item_ridx + user_key_word == 1

    load_keyw(sys.argv[1])
    print_keyw(sys.argv[2])
